#!/usr/bin/env python
import spade, urllib2, time, sys, sqlite3, re, random, json
from bs4 import BeautifulSoup, Comment
from collections import deque

class hotelCrawler(spade.Agent.Agent):
	class informWhitepages(spade.Behaviour.OneShotBehaviour):	# inform service directory
		def _process(self):
			sd = spade.DF.ServiceDescription()
			sd.setName("hotel")
			sd.setType("crawler")
			dad = spade.DF.DfAgentDescription()
			dad.addService(sd)
			dad.setAID(self.myAgent.getAID())
			res = self.myAgent.registerService(dad)
			print "Informed Directory Services"
			dad = spade.DF.DfAgentDescription()
			ds = spade.DF.ServiceDescription()
			ds.setType("provider")
			ds.setName("hotelurl")
			dad.addService(ds)
			agents = self.myAgent.searchService(dad)
			if len(agents) > 0:
				request = spade.ACLMessage.ACLMessage()
				request.setPerformative("request")
				request.setOntology("hotel")
				request.addReceiver(random.choice(agents).getAID())
				self.myAgent.send(request)
			print "Asked for a URL"

	class askForHotelURL(spade.Behaviour.PeriodicBehaviour):	# ask for a hotel url to parse
		def _onTick(self):
			dad = spade.DF.DfAgentDescription()
			ds = spade.DF.ServiceDescription()
			ds.setType("provider")
			ds.setName("hotelurl")
			dad.addService(ds)
			agents = self.myAgent.searchService(dad)
			if len(agents) > 0:
				request = spade.ACLMessage.ACLMessage()
				request.setPerformative("request")
				request.setOntology("hotel")
				request.addReceiver(random.choice(agents).getAID())
				self.myAgent.send(request)
			print "Asked for a URL"

	class receiveHotelURL(spade.Behaviour.EventBehaviour):		# receive a request to crawl a hotel url
		def _process(self):
			print "Waiting for Message"
			msg = self._receive(True)
			_reviews_ = []
			basic_url = msg.getContent()
			print "Visiting Hotel: " + basic_url
			soop = BeautifulSoup(urllib2.urlopen("http://www.tripadvisor.com" + basic_url).read(),'html.parser')
			limit = int(soop.find('label',{'for': 'sortOrder'}).text.split()[0]) if soop.find('label',{'for': 'sortOrder'}) else 0
			if soop.find("div",{"id":"REVIEWS"}):
				for review in soop.find("div",{"id":"REVIEWS"}).findAll('div', id=re.compile('^review_')):
					#_reviews_.append(review.findAll('a')[1]['href'])
					links = review.findAll('a')
					_reviews_.append(links[1]['href']) if len(links)>1 else "/Aqua"
			for i in xrange(10,limit,10):
				url = "-".join(basic_url.split('-')[:4]) + "-or" + str(i) + "-" + "-".join(basic_url.split('-')[4:])
				print "Visiting Hotel: " + url
				soop = BeautifulSoup(urllib2.urlopen("http://www.tripadvisor.com"+url.strip()).read(),'html.parser')
				if soop.find("div",{"id":"REVIEWS"}):
					for review in soop.find("div",{"id":"REVIEWS"}).findAll('div', id=re.compile('^review_')):
						links = review.findAll('a')
						_reviews_.append(links[1]['href']) if len(links)>1 else "/Aqua"
			print "Expected: " + str(limit) + " reviews, but found: " + str(len(_reviews_))
			outgoing = msg.createReply()
			outgoing.setOntology("hotel")
			outgoing.setPerformative("inform")
			try: outgoing.setProtocol(basic_url)
			except: print "failed to set protocol"
			outgoing.setContent(json.dumps(_reviews_))
			self.myAgent.send(outgoing)
			dad = spade.DF.DfAgentDescription()
			ds = spade.DF.ServiceDescription()
			ds.setType("provider")
			ds.setName("hotelurl")
			dad.addService(ds)
			agents = self.myAgent.searchService(dad)
			if len(agents) > 0:
				request = spade.ACLMessage.ACLMessage()
				request.setPerformative("request")
				request.setOntology("hotel")
				request.addReceiver(random.choice(agents).getAID())
				self.myAgent.send(request)
			print "Reply sent."

	class refused(spade.Behaviour.Behaviour):		# receive a refusal a hotel url
		def _process(self):
			msg = None
			msg = self._receive(True,30)
			if msg: print "got a refusal"

	class cleanup(spade.Behaviour.Behaviour):		# clean up message queue
		def _process(self):
			msg = None
			msg = self._receive(True,30)
			if msg: print "got stray message"

	class timewait(spade.Behaviour.PeriodicBehaviour):
		def _onTick(self):
			pass


	def _setup(self):
		request_template = spade.Behaviour.ACLTemplate()
		request_template.setOntology("hotel")
		request_template.setPerformative("request")
		self.addBehaviour(self.informWhitepages(),None)
		self.addBehaviour(self.receiveHotelURL(),spade.Behaviour.MessageTemplate(request_template))
		#self.addBehaviour(self.askForHotelURL(15),None)
		refused_template = spade.Behaviour.ACLTemplate()
		refused_template.setOntology("hotel")
		refused_template.setPerformative("refuse")
		self.addBehaviour(self.refused(),spade.Behaviour.MessageTemplate(refused_template))
		#self.addBehaviour(self.timewait(30),None)
		self.setDefaultBehaviour(self.cleanup())

if __name__ == "__main__":
	try: name = sys.argv[1]
	except: name = "hotelcrawler"
	a = hotelCrawler(name+"@127.0.0.1","secret")
	print "Started Hotel Crawler with name: " + name
	a.start()
	Alive = True
	while Alive:
		try: time.sleep(1)
		except KeyboardInterrupt: Alive = False
	a.stop()
	sys.exit(0)


